#!/bin/bash
mysql_user=admin
mysql_passwd=!QAZ2wsx
ssh_user=mysql
ssh_pass=QWer12#$
ssh_port=22
# 主机测试宕网卡时长(s)
test_time=60
# 对宕网卡主机的探测周期(s)
probe_interval=5
dbscale_port=16310
mysql_port=16315
zookeeper_port=16309
# 非功能测试集群节点ip
mysql_cmd_host=$1
#mysql_cmd_host=172.17.139.152
# 不使用nc命令
nc_cmd=close

function mysql_cmd(){
# mysql_cmd_port默认值为dbscale_port
[ ${#mysql_cmd_port} -eq 0 ] && mysql_cmd_port=$dbscale_port
#[ ${#mysql_cmd_host} -eq 0 ] && mysql_cmd_host=127.0.0.1
timeout 5 mysql -u$mysql_user -p$mysql_passwd  -h$mysql_cmd_host -P$mysql_cmd_port -e "$1" 2>/dev/null
}
function ssh_cmd(){
  [ ${#ssh_ip} -eq 0 ] && ssh_ip=$mysql_cmd_host
  #timeout 30 sshpass -p $ssh_pass ssh -p $ssh_port ${ssh_user}@${ssh_ip} -o StrictHostKeyChecking=no "source /etc/profile && $1" 2>/dev/null
  timeout 30 sshpass -p $ssh_pass ssh -p $ssh_port ${ssh_user}@${ssh_ip} -o StrictHostKeyChecking=no "source /etc/profile && $1"
}
# 查询MySQL各节点身份
function query_mysqlcluster_info(){
  echo "=======================确认MySQL集群节点信息======================="
num=0
mysql_cmd  'dbscale show dataservers;' | awk 'NR>1' > $0_cluster_info_tmp.txt
if [ `cat $0_cluster_info_tmp.txt | wc -l` -eq 0 ];then 
  echo "获取集群信息失败.."
  exit
fi
while read line
do
  echo 当前校验行为$line
  if [[ $line =~ 'Master_Online' ]] && [[ $line =~ "Server normal" ]];then
    mysql_master_info=`echo $line | awk '{print $2,$3}'`
    mysql_master=`echo $line | awk '{print $2}'`
    #echo 当前行为主$mysql_master
  elif [[ $line =~ 'Not_A_Master' ]] && [[ $line =~ "Slave normal" ]];then
    mysql_slave="$mysql_slave `echo $line | awk '{print $2}'`"
    #echo 当前行为从$mysql_slave
  else
    echo -e "`echo $line | awk '{print $2}'`数据节点状态异常"
    mysql_cmd  'dbscale show dataservers;'
    exit
  fi
  let num++
done<$0_cluster_info_tmp.txt
echo -e "mysql_master:$mysql_master\nmysql_slave:$mysql_slave"
}

# 校验mysql master和主从线程
function query_mysql_master(){
  echo "=======================验证MySQL主节点和主从线程======================="
  cluster_info=$(mysql_cmd 'dbscale show dataservers;' | awk 'NR>1&&NR<5 {print $2,$3}')
  for mysql_host in $cluster_ips
  do
    mysql_cmd_host=$mysql_host
    mysql_cmd_port=$(echo $cluster_info | xargs -n2 | awk '/'$mysql_host'/ {print $2}')
    Master_Host=$(mysql_cmd 'show slave status\G' | awk '/Master_Host/ {print $2}')
    host_gtid=$(mysql_cmd 'show master status;' | awk '/bin/ {print $NF}')
    gtid_info="$mysql_host $host_gtid $gtid_info"

    if [ -n "$Master_Host" ] && [ $Master_Host == $(echo $mysql_master_info | awk '{print $1}') ]; then
      # 查看主从状态
      IO_status=$(mysql_cmd 'show slave status\G' | awk '/Slave_IO_Running:/ {print $2}')
      SQL_status=$(mysql_cmd 'show slave status\G' | awk '/Slave_SQL_Running:/ {print $2}')
      if [ $IO_status == "Yes" ] && [ $SQL_status == "Yes" ]; then
        echo "$mysql_host主从线程正常..."
      else
        echo "请检查$mysql_host主从线程!!!"
        exit
      fi
    elif [ $mysql_cmd_host != $(echo $mysql_master_info | awk '{print $1}') ]; then
      #    echo "$mysql_host为主,无主从线程."
      #  else
      echo "请检查$mysql_host节点主从!!!"
      exit
    fi
  done
  #echo gtid_info$gtid_info mysql_master$mysql_master
  #echo -e "集群GTID信息为:\n`echo $gtid_info | xargs -n2`"
  master_gtid=$(echo $gtid_info | xargs -n2 | awk '/'$mysql_master'/ {print $NF}')
  for gtid in $(echo $gtid_info | xargs -n2 | awk '{print $NF}'); do
    if [ $gtid != $master_gtid ]; then
      echo "$(echo $gtid_info | xargs -n2 | awk '/'$gtid'/ {print $1}')节点gtid与主节点$mysql_master不一致."
      echo -e "集群GTID信息为:\n$(echo $gtid_info | xargs -n2)"
      exit
    fi
  done
}

# 确定dbscale master
function query_dbscale_master(){
  echo "=======================确认dbscale集群节点信息======================="
mysql_cmd_port=$dbscale_port
dbscale_master_info=`mysql_cmd 'dbscale request cluster info;'| grep -w ^master | awk '{print $3}'`
if [ ${#dbscale_master_info} -eq 0 ];then
  echo "dbscale异常,未获取到dbscale master..."
  exit
fi
#echo "dbscale_master_info:$dbscale_master_info"
dbscale_master=${dbscale_master_info%:*}
echo "dbscale_master: $dbscale_master"
}

# 确定zookeeper角色
function query_zookeeper_role(){
  echo "=======================确认zookeeper角色信息======================="
#zookeeper_cmd_dir=`ssh_cmd "ps -ef | grep -v grep | grep zookeeper | sed -rn 's#.*:(/.*/)app(/.*)/bin/.*#\1app\2/bin#p'"`
for ssh_ip in $cluster_ips
do
  if [ "$nc_cmd" == 'close' ];then
    ssh_cmd "netstat -tunlp 2>/dev/null| grep 2888"1>/dev/null
    if [[ $? -eq 0 ]];then
      zookeeper_leader=$ssh_ip
      echo "zookeeper_leader: $zookeeper_leader"
      continue
    fi
    ssh_cmd "netstat -tunlp 2>/dev/null| grep 3888"1>/dev/null
    if [ $? -eq 0 ];then
      zookeeper_follower=$ssh_ip
      echo "zookeeper_follower: $zookeeper_follower"
    else
      echo "$ssh_ip执行netstat -tunlp 2>/dev/null| grep 3888 未获取到3888端口"
      exit
    fi
  else
    #zookeeper_role=`ssh_cmd  "sudo systemctl restart configuration.service && ${zookeeper_cmd_dir}/zkServer.sh status" 2>/dev/null | awk '/Mode:/ {print $2}' `
    #ssh_cmd  "echo stat | nc 127.0.0.1 16309 | grep -i mode | awk '{print \$2}'"
    zookeeper_role=`ssh_cmd  'echo stat | nc 127.0.0.1 16309 | grep -i mode '| awk '{print $2}'`
    if [[ $zookeeper_role == 'leader' ]];then
      zookeeper_leader=$ssh_ip
      echo "zookeeper_leader: $zookeeper_leader"
    elif [[ $zookeeper_role == 'follower' ]];then
      zookeeper_follower=$ssh_ip
      echo "zookeeper_follower: $zookeeper_follower"
    else
      echo "$ssh_ip执行echo stat | nc 127.0.0.1 16309 | grep -i mode | awk '/Mode:/ {print \$2}'无法获取到状态,请检查nc命令和zookeeper服务!!!"
      exit
    fi
  fi
done
}
function check_cluster_info(){
  query_mysqlcluster_info
  query_mysql_master
}
# 获取网卡名称和生成宕网卡脚本
function get_network_name(){
  echo "=======================生成各个节点宕网卡脚本======================="
 # echo $cluster_ips
for ssh_ip in $cluster_ips
do
  #echo $ssh_ip
  network_names=`ssh_cmd ifconfig | grep '^[^ lo]' | awk -F: '{print $1}'`
  for network_name in $network_names
  do
    ssh_cmd "ip a s $network_name" | grep $ssh_ip >/dev/null
   if [ $? -eq 0 ];then
#     echo network_name:$network_name
    echo "$ssh_ip节点宕网卡脚本:"
    ssh_cmd "echo -e 'echo NETWORK_CARD BEGIN STOP\ndate\necho -------------------------\nifdown $network_name\nsleep ${test_time}\nifup $network_name\necho -------------------------\necho NETWORK_CARD BEGIN START FINISH\ndate\n' > /home/mysql/networkdown.sh && cat /home/mysql/networkdown.sh"
   fi
  done
done
} 
# 组件重启命令
function restart_service(){
  #echo "=======================开始重启服务======================="
  mysql_cmd_host=$ssh_ip
  if [ $1 == "zookeeper" ];then
    echo "$zookeeper_leader节点zookeeper重启....."
    ssh_ip=$zookeeper_leader
    #ssh_cmd "$zookeeper_cmd_dir/zkServer.sh restart && sleep 5"
   # check_count=0
   # while true
   # do
   #         ssh_cmd "ps -ef | grep zookeeper" >/dev/null
   #         if [ $? -eq 0 ];then
   #           break
   # 	    else
   #           let check_count++
   #           sleep 2
   #         fi
   # done
    zk_pid=`ssh_cmd 'ps -ef | grep zookeeper | grep -v grep'| awk '{print $2}'`
    echo zk_pid:$zk_pid
    ssh_cmd "kill -9 $zk_pid && sudo systemctl stop configuration.service && /data/app/zookeeper-3.6.2/bin/zkServer.sh start"
    #ssh_cmd "kill -9 $zk_pid && sudo systemctl restart configuration.service"
    #ssh_cmd 'kill -9 `ps -ef | grep zookeeper | grep -v grep | awk '{print $2}'` && sudo systemctl restart configuration.service'
    #echo "$? 更换zk身份"
    if [ $? -eq 0 ];then
      sleep 5
      query_zookeeper_role
    else
      echo "zookeeper重启失败,请检查zookeeper服务!!!"
    fi
  elif [ $1 == "dbscale" ];then
    echo "$dbscale_master节点dbscale重启....."
    ssh_ip=$dbscale_master
    ssh_cmd "cd /data/app/dbscale && ./daemon/stopper.sh && sleep 5"
    echo 停止命令回执$?
    ssh_cmd "cd /data/app/dbscale &&  ./daemon/dbscale_daemon.py"
    echo 启动命令回执$?
    while true
    do
      sleep 5
      mysql_cmd_host=$dbscale_master
      mysql_cmd 'dbscale show dataservers;' >/dev/null
      #echo 测试回执$?
      if [ $? -eq 0 ];then
        mysql_cmd 'dbscale request cluster info;'
        sleep 5
        break
      else
        echo "dbscale启动中..."
      fi
    done
    query_dbscale_master
  fi
}

# 调整环境
function change_master(){
   echo "=======================环境准备中======================="
   count=0
    while true
    do
      if [ $key == "mysql" ];then
        if [ $mysql_master == $dbscale_master ];then
            restart_service dbscale
        elif [ $mysql_master == $zookeeper_leader ];then
            restart_service zookeeper
        else
          break
        fi
      elif [ $key == "dbscale" ];then
        if [ $dbscale_master == $mysql_master ];then
            restart_service dbscale
        elif [ $dbscale_master == $zookeeper_leader ];then
            restart_service zookeeper
        else
          break
        fi
      elif [ $key == "zookeeper" ];then
        if [ $zookeeper_leader == $mysql_master ];then
            restart_service zookeeper
        elif [ $zookeeper_leader == $dbscale_master ];then
            restart_service dbscale
        else
          break
        fi
      fi
     # sleep 30
      let count++
      if [ $count -gt 2 ];then
        echo "$key节点环境准备失败..."
        exit
      fi
    done
}

function check_host(){
    echo "=======================开始探测主机======================="
    count=0
    while true
    do
        ping -c 1 $down_network_node_ip >/dev/null 2>&1
        current_time=$(date +%s)
        check_time=$((current_time - down_network_time))
        if [ $? -eq 0 ];then
            if [ $check_time -le $((test_time/2)) ];then
                 echo -e "宕网卡可能失败,检测时间$check_time,小于$((test_time/2)),具体时间以最后输出的日志为主.\n当网卡命令为:\ntimeout 5 sshpass -p $ssh_pass ssh -p $ssh_port ${ssh_user}@$down_network_node_ip -o StrictHostKeyChecking=no 'source /etc/profile && nohup sudo sh /home/mysql/networkdown.sh >> /home/mysql/networkdown.log &'"
            fi
            ssh_ip=$down_network_node_ip
            echo -e "=======================获取宕网卡日志信息=======================\n$(ssh_cmd 'cat /home/mysql/networkdown.log')"
            break
        else
                count=$((count + 1))
        fi
        if [ $check_time -ge $test_time ];then
            echo "主机启动异常,已宕机约${check_time}秒."
            break
        elif [ $((count % 3)) -eq 0 ]; then
            echo -e "当前时间:$(date +%T),已宕机约${check_time}s"
        fi
        sleep $probe_interval
    done
    exit
}

# 宕节点网卡
function down_network_node(){
  echo "=======================开始宕网卡======================="
    echo -e "mysql_master:$mysql_master\ndbscale_master:$dbscale_master\nzookeeper_leader:$zookeeper_leader"
    echo "执行宕网卡节点为:$down_network_node_ip"
    read -p "请确认宕网卡节点($key)信息是否正确(y/n):" choose2
      if [[ $choose2 == 'n' ]];then
        echo "退出..."
        exit
      elif [[ $choose2 != 'y' ]];then
        echo "请正确输入选择!!!"
        exit
      fi
      #timeout 5 sshpass -p $ssh_pass ssh -p $ssh_port ${ssh_user}@$down_network_node_ip -o StrictHostKeyChecking=no "source /etc/profile && echo '$ssh_ip节点宕网卡时间为:`date +%T`' && nohup sudo sh /home/mysql/networkdown.sh &" 2>/dev/null
      timeout 20 sshpass -p $ssh_pass ssh -p $ssh_port ${ssh_user}@$down_network_node_ip -o StrictHostKeyChecking=no "source /etc/profile && nohup sudo sh /home/mysql/networkdown.sh > /home/mysql/networkdown.log &"
    #   cmd_result=$?
    down_network_time=`date -d '10 seconds ago' '+%s'`
    #   echo "宕网卡命令返回值$cmd_result `date +%T`"
    #   if [ $cmd_result -ne 0 ];then
    #     echo -e "宕网卡失败,请检查命令:\ntimeout 25 sshpass -p $ssh_pass ssh -p $ssh_port ${ssh_user}@$down_network_node_ip -o StrictHostKeyChecking=no 'source /etc/profile && nohup sudo sh /home/mysql/networkdown.sh >> /home/mysql/networkdown.log &'"
    #     exit
    #   fi
      echo "$down_network_node_ip节点宕网卡时间约为:`date -d @$down_network_time +%T`"
      service_recovery_time=$[down_network_time + test_time]
      echo "网络预计恢复时间为 `date -d @$service_recovery_time +%T`"
      #sleep 5
      if [ $key == "zookeeper" ];then
        check_host
      else
        error_num=0
        echo 检验节点为:$zookeeper_leader
        while true
        do
            #mysql_cmd_host=`echo $mysql_slave | awk '{print $1}' `
             mysql_cmd_host=$zookeeper_leader
             mysql_cmd_port=$dbscale_port
             mysql_cmd "show databases;"&>/dev/null
             if [ $? -eq 0 ];then
               service_recovery_time=`date +%s`
               echo "dbscale业务恢复时间约为:`date -d @$service_recovery_time +%T` "
               echo "dbscale服务中断约$[service_recovery_time-down_network_time]s"
               if [ $[service_recovery_time-down_network_time] -lt 60 ];then
                 echo "主从切换时间低于一分钟"
               else
                 echo "主从切换时间不满足需求"
               fi
                mysql_cmd 'dbscale show dataservers;dbscale request cluster info'
                check_host
             else
              let error_num++
              if [ $error_num -eq 10 ];then
                echo "`date +%T`:主从切换中..."
                error_num=0
              fi
             fi
        done
      fi
}
function get_cluster_ips(){
  cluster_ips=`mysql_cmd 'dbscale show dataservers\G' | awk '/host/{print $2}'`
  [ ${#cluster_ips} -eq 0 ] && echo "连接${mysql_cmd_host}dbscale失败!!!" && exit
  for ip in $cluster_ips
  do
    mysql_cmd_host=$ip
    mysql_cmd 'dbscale show dataservers\G' >/dev/null
    if [ $? -ne 0 ];then
      echo $mysql_cmd_host节点dbscale异常
      exit
    fi
  done
}

function check_template(){
  echo -e "\e[0;32m==================$1==================\e[0m"
  for i in $cluster_ips
  do
    ssh_ip=$i
    #ssh_cmd "$2"
    $2
    if [ $? -eq 0 ];then
      echo "$i $3"
    else
      echo -e "\e[0;31m$i $4\e[0m"
    fi
    sleep 1
  done
}

function check(){
  get_cluster_ips
  check_template "check_ssh_pass" "exit" "ssh_pass check success!!!" "ssh_pass check fail!!!"
  check_template "check_sudo_privilege" "sudo -n true" "sudo_privilege check success!!!" "sudo_privilege check fail!!!"
  if [ "$nc_cmd" == "close" ];then
    check_template "check_service_port" "(echo>/dev/tcp/$i/$dbscale_port)&>/dev/null && (echo>/dev/tcp/$i/$mysql_port)&>/dev/null && (echo>/dev/tcp/$i/$zookeeper_port)&>/dev/null" "service_port check success!!!" "service_port check fail!!!"
  else
    check_template "check_service_port" "nc -vz 127.0.0.1 $dbscale_port && nc -vz 127.0.0.1 $mysql_port && nc -vz 127.0.0.1 $zookeeper_port" "service_port check success!!!" "service_port check fail!!!"
  fi
}

function exec_list(){
  		    query_mysqlcluster_info
	        query_mysql_master
	        query_zookeeper_role
	        query_dbscale_master
	        get_network_name
	        change_master
}


if [ $# != 2 ] || ! [[ $1 =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]];then
    echo -e  "USAGE:(sh target_cluster_ip mysql|zookeeper|dbscale|check\nEXAMPLES:\n \t宕mysql主节点: sh $0 集群任意ip mysql"
    exit
fi

# 确定集群ip列表
get_cluster_ips
echo -e "cluster_ip_list:\n\t `echo $cluster_ips`"

read -p "请确认集群ip是否一致(y/n):" choose1
[ $choose1 != y ] && exit

key=$2
#case $2 in
case $key in
        mysql)
          exec_list
	        down_network_node_ip=$mysql_master
	        down_network_node
        ;;
        zookeeper)
          exec_list
          down_network_node_ip=$zookeeper_leader
	        down_network_node
        ;;
        dbscale)
          exec_list
          down_network_node_ip=$dbscale_master
	        down_network_node
       ;;
       check)
          check
       ;;
        *)
          echo -e "请遵照提示使用脚本...\nUSAGE:(target_cluster_ip mysql|zookeeper|dbscale|check)"
 esac